package cnki.fetch;

import cnki.db.sql.Operator;
import cnki.entity.PeriodicalInfoEntity;
import cnki.proxy.ProxyImpl;
import cnki.util.DataWasher;
import cnki.util.FileManager;
import cnki.util.ListTool;
import com.han.http.client.exception.NullContentMatchedException;
import com.han.http.util.Regex;
import org.json.JSONArray;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.io.IOException;
import java.net.Proxy;
import java.sql.Timestamp;
import java.util.*;
import java.util.concurrent.Executors;

import static com.han.http.util.Print.printlnErr;
import static com.han.http.util.Print.printlnInfo;

/**
 * @author: Han
 * @date: 2018/9/27 9:44
 * @dscription: 根据期刊名搜索期刊代码
 */
public class PeriodicalFetcher {
    public static void main(String... args) {
        var fetcher = new PeriodicalFetcher();
        fetcher.getPeriodicalCatalog();
    }

    private void getPeriodicalBaseInfo() {
        var periodicalList = DataWasher.washPeriodicalCatalog();
        var splitList = new ListTool<String>().splitListByCount(periodicalList, 10);
        var executor = Executors.newFixedThreadPool(10);
        var jMap = new TreeMap<String, PeriodicalInfoEntity>();
        for (var i = 0; i < splitList.size(); i++) {
            var tempList = splitList.get(i);
            var thread = new PeriodicalBaseThread(tempList, jMap);
            thread.setName("Thread-基础信息-" + i);
            executor.submit(thread);
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        executor.shutdown();
        //等待线程池线程运行结束,去充值后同时插入
        while (!executor.isTerminated()) {
            try {
                Thread.sleep(1000);
                printlnInfo("仍在继续:" + Thread.activeCount());
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        var periodicalInfoList = jMap.values();
        var operator = Operator.getInstance();
        for (var infoEntity : periodicalInfoList) {
            operator.insertPeriodicalInfoEntity(infoEntity);
        }
    }

    private void getPeriodicalCatalog() {
        var operator = Operator.getInstance();
        var periodicalInfoEntityList = operator.queryPeriodicalInfo();
        var splitList = new ListTool<PeriodicalInfoEntity>().splitListByCount(periodicalInfoEntityList, 10);
        var executor = Executors.newFixedThreadPool(10);
        for (int i = 0; i < splitList.size(); i++) {
            var tempList = splitList.get(i);
            PeriodicalCatalogThread thread = new PeriodicalCatalogThread(tempList);
            thread.setName("Thread-目录抓取-" + i);
            executor.submit(thread);
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
        executor.shutdown();
    }
}

/**
 * 抓取期刊基本信息
 */
class PeriodicalBaseThread extends Thread {
    private FileManager fileManager;
    private List<String> nameList;
    private Proxy httpProxy;
    private ProxyImpl proxyImpl;
    private Map<String, PeriodicalInfoEntity> jMap;

    PeriodicalBaseThread(List<String> nameList, Map<String, PeriodicalInfoEntity> jMap) {
        this.nameList = nameList;
        this.jMap = jMap;
        proxyImpl = new ProxyImpl();
        httpProxy = proxyImpl.getHttpProxy();
        fileManager = new FileManager(PeriodicalBaseThread.class);
    }

    @Override
    public void run() {
        var repeatCount = 0;
        var random = new Random();
        for (var i = 0; i < nameList.size(); i++) {
            var journalName = nameList.get(i);
            try {
                genJournalBaseInfo(journalName);
                //抓取成功充值重复计数
                repeatCount = 0;
                fileManager.write(journalName, true);
            } catch (IOException e) {
                //如果抓取不成功,换代理重试
                printlnErr(getName() + " " + httpProxy + "不可用,换");
                httpProxy = proxyImpl.getHttpProxy();
                //同一个期刊名最多重复十次
                if (repeatCount++ < 10) {
                    i--;
                } else {
                    fileManager.write(journalName, true);
                    printlnErr(getName() + " 获取失败" + journalName);
                }
            } finally {
                try {
                    //最多休眠200毫秒,最少休眠100毫秒
                    var interval = random.nextInt(100) + 100;
                    Thread.sleep(interval);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 获取期刊基本信息
     *
     * @param journalName 期刊名称
     * @throws IOException 抛出异常
     */
    private void genJournalBaseInfo(String journalName) throws IOException {
        //请求头
        var headers = new HashMap<String, String>(5);
        headers.put("Accept", "text/plain, */*; q=0.01");
        headers.put("X-Requested-With", "XMLHttpRequest");
        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3534.4 Safari/537.36");
        headers.put("Content-Type", "application/x-www-form-urlencoded");
        headers.put("Accept-Encoding", "gzip, deflate");
        headers.put("Accept-Language", "en,zh-CN;q=0.9,zh;q=0.8");

        //Post数据
        var datas = new HashMap<String, String>(5);
        datas.put("SearchStateJson", "{\"StateID\":\"\",\"Platfrom\":\"\",\"QueryTime\":\"\",\"Account\":\"knavi\",\"ClientToken\":\"\",\"Language\":\"\",\"CNode\":{\"PCode\":\"CJFD\",\"SMode\":\"\",\"OperateT\":\"\"},\"QNode\":{\"SelectT\":\"\",\"Select_Fields\":\"\",\"S_DBCodes\":\"\",\"QGroup\":[{\"Key\":\"subject\",\"Logic\":1,\"Items\":[],\"ChildItems\":[{\"Key\":\"txt\",\"Logic\":1,\"Items\":[{\"Key\":\"txt_1\",\"Title\":\"\",\"Logic\":1,\"Name\":\"TI\",\"Operate\":\"%\",\"Value\":\"'" + journalName + "'\",\"ExtendType\":0,\"ExtendValue\":\"\",\"Value2\":\"\"}],\"ChildItems\":[]}]}],\"OrderBy\":\"OTA|DESC\",\"GroupBy\":\"\",\"Additon\":\"\"}}");
        datas.put("displaymode", "1");
        datas.put("pageindex", "1");
        datas.put("pagecount", "50");
        datas.put("index", "1");
        datas.put("random", "0.9181720605280197");

        var url = "http://navi.cnki.net/knavi/Common/Search/Journal";
        var connection = Jsoup.connect(url);
        connection.timeout(1000 * 10);
        connection.headers(headers);
        connection.data(datas);
        connection.proxy(httpProxy);
        try {
            var document = connection.post();
            parseBody(document);
        } catch (IOException e) {
            throw e;
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 解析HTML页面
     *
     * @param document 页面内容
     */
    private void parseBody(Document document) {
        //获取期刊内容
        var listElement = document.getElementsByClass("list_tup").first();
        //获取期刊内容列表
        var lis = listElement.getElementsByTag("li");
        for (var tempEle : lis) {
            try {
                var aTag = tempEle.select("a").first();
                var journalName = aTag.attr("title");
                //解析href,pubtype,pcode,baseid
                var href = aTag.attr("href").replaceAll("[\\s]", "");
                var pubtype = "";
                try {
                    pubtype = Regex.matchString(href, "(?<=pubtype=).*(?=pcode)");
                } catch (NullContentMatchedException ignore) {
                }
                var pcode = "";
                try {
                    pcode = Regex.matchString(href, "(?<=pcode=).*(?=baseid)");
                } catch (NullContentMatchedException ignore) {
                }
                var baseid = "";
                try {
                    baseid = Regex.matchString(href, "(?<=baseid=).*");
                } catch (NullContentMatchedException ignore) {
                }
                href = "http://navi.cnki.net" + href;
                //解析
                var detail = aTag.getElementsByClass("detials").first().toString();
                var cif = "";
                try {
                    cif = Regex.matchString(detail, "(?<=复合影响因子：)((?!<).*)(?=<)");
                } catch (NullContentMatchedException ignore) {
                }
                var aif = "";
                try {
                    aif = Regex.matchString(detail, "(?<=综合影响因子：)((?!<).*)(?=<)");
                } catch (NullContentMatchedException ignore) {
                }
                var issn = "";
                try {
                    issn = Regex.matchString(detail, "(?<=ISSN：)((?!<).*)(?=<)");
                } catch (NullContentMatchedException ignore) {
                }
                var cn = "";
                try {
                    cn = Regex.matchString(detail, "(?<=CN：)((?!<).*)(?=<)");
                } catch (NullContentMatchedException ignore) {
                }
                var sponsor = "";
                try {
                    sponsor = Regex.matchString(detail, "(?<=主办单位：)((?!<).*)(?=<)");
                } catch (NullContentMatchedException ignore) {
                }
                var currentTime = new Timestamp(System.currentTimeMillis());
                var entity = new PeriodicalInfoEntity(journalName, href, pubtype, pcode, baseid, "",
                        cif, aif, issn, cn, sponsor, currentTime, currentTime);
                //不存储重名期刊
                jMap.put(journalName, entity);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}

class PeriodicalCatalogThread extends Thread {
    private FileManager fileManager;
    private Operator operator;
    private Proxy httpProxy;
    private ProxyImpl proxyImpl;
    private List<PeriodicalInfoEntity> periodicalInfoEntityList;

    PeriodicalCatalogThread(List<PeriodicalInfoEntity> periodicalInfoEntityList) {
        this.periodicalInfoEntityList = periodicalInfoEntityList;
        operator = Operator.getInstance();
        proxyImpl = new ProxyImpl();
        httpProxy = proxyImpl.getHttpProxy();
        fileManager = new FileManager(PeriodicalCatalogThread.class);
    }

    @Override
    public void run() {

        var repeatCount = 0;
        var random = new Random();
        for (var i = 0; i < periodicalInfoEntityList.size(); i++) {
            var journalName = periodicalInfoEntityList.get(i);
            try {
                getCatalog(journalName);
                //抓取成功充值重复计数
                repeatCount = 0;
                fileManager.write(journalName.toString(), true);
            } catch (IOException e) {
                //如果抓取不成功,换代理重试
                printlnErr(getName() + " " + httpProxy + "不可用,换");
                httpProxy = proxyImpl.getHttpProxy();
                //同一个期刊名最多重复十次
                if (repeatCount++ < 10) {
                    i--;
                } else {
                    fileManager.write(journalName.toString(), true);
                    printlnErr(getName() + " 获取失败" + journalName);
                }
            } finally {
                try {
                    //最多休眠200毫秒,最少休眠100毫秒
                    var interval = random.nextInt(100) + 100;
                    Thread.sleep(interval);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    private void getCatalog(PeriodicalInfoEntity entity) throws IOException {
        var headers = new HashMap<String, String>(5);
        headers.put("Host", "navi.cnki.net");
        headers.put("Connection", "keep-alive");
        headers.put("Accept", "*/*");
        headers.put("X-Requested-With", "XMLHttpRequest");
        headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3534.4 Safari/537.36");
        headers.put("Referer", "http://navi.cnki.net/knavi/JournalDetail?pcode=CJFD&pykm=QHDZ");
        headers.put("Accept-Encoding", "gzip, deflate");
        headers.put("Accept-Language", "en,zh-CN;q=0.9,zh;q=0.8");

        String baseid = entity.getBaseid();
        var url = "http://navi.cnki.net/knavi/JournalDetail/GetJournalYearList?pcode=CJFD&pykm=" + baseid + "&pIdx=0";
        var connection = Jsoup.connect(url);
        connection.timeout(1000 * 10);
        connection.headers(headers);
        connection.proxy(httpProxy);
        try {
            var document = connection.get();
            var issue = parseBody(document);
            entity.setIssue(issue);
            operator.updatePeriodicalInfoIssue(entity);
        } catch (IOException e) {
            throw e;
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 解析document,获取期刊目录并生成json形式数据
     *
     * @param document 目录页面内容
     */
    private String parseBody(Document document) {
        //总框架
        var dtObj = new JSONObject();
        //年份刊号所有页码集合
        var yearsIssuePages = document.getElementsByClass("yearissuepage");
        //遍历每一页
        for (var i = yearsIssuePages.size() - 1; i >= 0; i--) {
            var yearsIssuePage = yearsIssuePages.get(i);
            //将每一页中的年份刊号取出
            var yearIssuePages = yearsIssuePage.getElementsByTag("dl");
            //遍历每个年份刊号
            for (var j = yearIssuePages.size() - 1; j >= 0; j--) {
                var yearIssuePage = yearIssuePages.get(j);
                var dtEle = yearIssuePage.getElementsByTag("dt").first();
                var dt = dtEle.text();
                var ddArray = new JSONArray();
                var ddEles = yearIssuePage.getElementsByTag("dd").select("a");
                for (var k = ddEles.size() - 1; k >= 0; k--) {
                    var ddEle = ddEles.get(k);
                    var dd = ddEle.text();
                    //每一个dd生成条记录,并标记其是否被抓去过
                    var ddObj = new JSONObject();
                    ddObj.put("dd", dd);
                    ddObj.put("fetched", false);
                    ddArray.put(ddObj);
                }
                dtObj.put(dt, ddArray);
            }
        }
        return dtObj.toString();
    }
}